{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\python36\\lib\\gensim\\utils.py:865: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "import math\n",
    "from string import punctuation\n",
    "from heapq import nlargest\n",
    "from itertools import product, count\n",
    "from gensim.models import word2vec\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'divide': 'warn', 'over': 'warn', 'under': 'ignore', 'invalid': 'warn'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = word2vec.Word2Vec.load(\"word2vec/word2vec_wx\")\n",
    "np.seterr(all='warn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cut_sentences(sentence):\n",
    "    puns = frozenset(u'。！？')\n",
    "    tmp = []\n",
    "    for ch in sentence:\n",
    "        tmp.append(ch)\n",
    "        if puns.__contains__(ch):\n",
    "            yield ''.join(tmp)\n",
    "            tmp = []\n",
    "    yield ''.join(tmp)\n",
    " \n",
    " \n",
    "# 句子中的stopwords\n",
    "def create_stopwords():\n",
    "    stop_list = [line.strip() for line in open(\"stopwords.txt\", 'r', encoding='utf-8').readlines()]\n",
    "    return stop_list\n",
    " \n",
    " \n",
    "def two_sentences_similarity(sents_1, sents_2):\n",
    "    '''\n",
    "    计算两个句子的相似性\n",
    "    :param sents_1:\n",
    "    :param sents_2:\n",
    "    :return:\n",
    "    '''\n",
    "    counter = 0\n",
    "    for sent in sents_1:\n",
    "        if sent in sents_2:\n",
    "            counter += 1\n",
    "    return counter / (math.log(len(sents_1) + len(sents_2)))\n",
    " \n",
    " \n",
    "def create_graph(word_sent):\n",
    "    \"\"\"\n",
    "    传入句子链表  返回句子之间相似度的图\n",
    "    :param word_sent:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    num = len(word_sent)\n",
    "    board = [[0.0 for _ in range(num)] for _ in range(num)]\n",
    " \n",
    "    for i, j in product(range(num), repeat=2):\n",
    "        if i != j:\n",
    "            board[i][j] = compute_similarity_by_avg(word_sent[i], word_sent[j])\n",
    "    return board\n",
    " \n",
    " \n",
    "def cosine_similarity(vec1, vec2):\n",
    "    '''\n",
    "    计算两个向量之间的余弦相似度\n",
    "    :param vec1:\n",
    "    :param vec2:\n",
    "    :return:\n",
    "    '''\n",
    "    tx = np.array(vec1)\n",
    "    ty = np.array(vec2)\n",
    "    cos1 = np.sum(tx * ty)\n",
    "    cos21 = np.sqrt(sum(tx ** 2))\n",
    "    cos22 = np.sqrt(sum(ty ** 2))\n",
    "    cosine_value = cos1 / float(cos21 * cos22)\n",
    "    return cosine_value\n",
    " \n",
    " \n",
    "def compute_similarity_by_avg(sents_1, sents_2):\n",
    "    '''\n",
    "    对两个句子求平均词向量\n",
    "    :param sents_1:\n",
    "    :param sents_2:\n",
    "    :return:\n",
    "    '''\n",
    "    if len(sents_1) == 0 or len(sents_2) == 0:\n",
    "        return 0.0\n",
    "    if sents_1[0] in model:\n",
    "        \n",
    "        vec1 = model[sents_1[0]]\n",
    "    else:\n",
    "        vec1=model['中国']\n",
    "    for word1 in sents_1[1:]:\n",
    "        if word1 in model:\n",
    "            \n",
    "            vec1 = vec1 + model[word1]\n",
    "        else:\n",
    "            vec1 = vec1 + model['中国']\n",
    "            \n",
    "    if sents_2[0] in model:\n",
    "        \n",
    "        vec2 = model[sents_2[0]]\n",
    "    else:\n",
    "        vec2=model['中国']\n",
    "    for word2 in sents_2[1:]:\n",
    "        if word2 in model:\n",
    "            \n",
    "            vec2 = vec2 + model[word2]\n",
    "        else:\n",
    "            vec2 = vec2 + model['中国']\n",
    " \n",
    "    similarity = cosine_similarity(vec1 / len(sents_1), vec2 / len(sents_2))\n",
    "    return similarity\n",
    " \n",
    " \n",
    "def calculate_score(weight_graph, scores, i):\n",
    "    \"\"\"\n",
    "    计算句子在图中的分数\n",
    "    :param weight_graph:\n",
    "    :param scores:\n",
    "    :param i:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    length = len(weight_graph)\n",
    "    d = 0.85\n",
    "    added_score = 0.0\n",
    " \n",
    "    for j in range(length):\n",
    "        fraction = 0.0\n",
    "        denominator = 0.0\n",
    "        # 计算分子\n",
    "        fraction = weight_graph[j][i] * scores[j]\n",
    "        # 计算分母\n",
    "        for k in range(length):\n",
    "            denominator += weight_graph[j][k]\n",
    "            if denominator == 0:\n",
    "                denominator = 1\n",
    "        added_score += fraction / denominator\n",
    "    # 算出最终的分数\n",
    "    weighted_score = (1 - d) + d * added_score\n",
    "    return weighted_score\n",
    " \n",
    " \n",
    "def weight_sentences_rank(weight_graph):\n",
    "    '''\n",
    "    输入相似度的图（矩阵)\n",
    "    返回各个句子的分数\n",
    "    :param weight_graph:\n",
    "    :return:\n",
    "    '''\n",
    "    # 初始分数设置为0.5\n",
    "    scores = [0.5 for _ in range(len(weight_graph))]\n",
    "    old_scores = [0.0 for _ in range(len(weight_graph))]\n",
    " \n",
    "    # 开始迭代\n",
    "    while different(scores, old_scores):\n",
    "        for i in range(len(weight_graph)):\n",
    "            old_scores[i] = scores[i]\n",
    "        for i in range(len(weight_graph)):\n",
    "            scores[i] = calculate_score(weight_graph, scores, i)\n",
    "    return scores\n",
    " \n",
    " \n",
    "def different(scores, old_scores):\n",
    "    '''\n",
    "    判断前后分数有无变化\n",
    "    :param scores:\n",
    "    :param old_scores:\n",
    "    :return:\n",
    "    '''\n",
    "    flag = False\n",
    "    for i in range(len(scores)):\n",
    "        if math.fabs(scores[i] - old_scores[i]) >= 0.0001:\n",
    "            flag = True\n",
    "            break\n",
    "    return flag\n",
    " \n",
    " \n",
    "def filter_symbols(sents):\n",
    "    stopwords = create_stopwords() + ['。', ' ', '.']\n",
    "    _sents = []\n",
    "    for sentence in sents:\n",
    "        for word in sentence:\n",
    "            if word in stopwords:\n",
    "                sentence.remove(word)\n",
    "        if sentence:\n",
    "            _sents.append(sentence)\n",
    "    return _sents\n",
    " \n",
    " \n",
    "def filter_model(sents):\n",
    "    _sents = []\n",
    "    for sentence in sents:\n",
    "        for word in sentence:\n",
    "            if word not in model:\n",
    "                \n",
    "                sentence.remove(word)\n",
    "        if sentence:\n",
    "            _sents.append(sentence)\n",
    "    return _sents\n",
    " \n",
    " \n",
    "def summarize(text, n):\n",
    "    tokens = cut_sentences(text)\n",
    "    sentences = []\n",
    "    sents = []\n",
    "    for sent in tokens:\n",
    "        sentences.append(sent)\n",
    "        sents.append([word for word in jieba.cut(sent) if word])\n",
    " \n",
    "    # sents = filter_symbols(sents)\n",
    "    sents = filter_model(sents)\n",
    "    graph = create_graph(sents)\n",
    " \n",
    "    scores = weight_sentences_rank(graph)\n",
    "    sent_selected = nlargest(n, zip(scores, count()))\n",
    "    sent_index = []\n",
    "    for i in range(n):\n",
    "        sent_index.append(sent_selected[i][1])\n",
    "    return [sentences[i] for i in sent_index]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TSM创始人Andy Dinh电竞发展迅猛，投资变成了必不可少的环节，Dinh称TSM运作精良，但也需要一个投资方来帮助TSM走得更远。\n",
      "********************\n",
      "库里与伊戈达拉投资TSMTSM排面十足 行业巨头纷纷投资TSM的创始人Andy Dinh此前一直自力更生，从TSM在2009年成立以来，他一人承担起了队员、教练、公关等一系列角色，现在因为这笔投资，他终于成功卸下一部分重担。\n",
      "********************\n",
      "2011年，TSM开始涉及英雄联盟项目，起初Reginald也是选手中的一员，他是队内的中单“猩猩队长”，作为队内的核心Carry点，他打法凶悍，擅长变阵，为队伍带起不少节奏。\n",
      "********************\n"
     ]
    }
   ],
   "source": [
    "with open(\"data/3.txt\", \"r\", encoding='gbk') as myfile:\n",
    "        text = myfile.read().replace('\\n', '')\n",
    "        summarys=summarize(text,3)\n",
    "        for each in summarys:\n",
    "            print (each+\"\\n\"+\"*\"*20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
